
import pymongo
import scrapy
from scrapy import Request
import sqlite3


class T66y(scrapy.Spider):
    name = 't66y_item'
    allowed_domains = ['t66y.com', 'filefab.com']

    def __init__(self, name=None, **kwargs):
        super().__init__(name=None, **kwargs)
        self.conn = sqlite3.connect('t66y.db')
        self.c = self.conn.cursor()
        self.mongo_client = pymongo.MongoClient(host='127.0.0.1', port=27017)
        self.collection = self.mongo_client.t66y.article

    def start_requests(self):
        yield Request('https://t66y.com/htm_data/2203/25/4993699.html', callback=self.parse,
                          dont_filter=True, meta={'proxy': 'http://127.0.0.1:58591'})

    def parse(self, response):
        details = response.xpath("//div[@class='tpc_content do_not_catch']//text()|//div[@class='tpc_content do_not_catch']//img/@ess-data")
        items = [item for item in details if "名称" in item.extract()]
        item = {'details': [], 'links': []}
        if len(items) > 1:
            struct = None
            for detail in details:
                value = detail.extract()
                print("value: " + str(value))
                if "名稱" in value or "名称" in value:
                    struct = {'name': value, 'imgs': []}
                    dict.setdefault(struct, 'imgs', [])
                    item['details'].append(struct)

                elif ("格式" in value or "格式" in value) and struct:
                    struct['format'] = value

                elif ("大小" in value or "大小" in value) and struct:
                    struct['size'] = value

                elif ("时长" in value or "时间" in value) and struct:
                    struct['time'] = value

                elif type(value) == str and (value.startswith('http') or value.startswith('https')):
                    if struct :
                        struct['imgs'].append(value)

            external_links = response.xpath("//a[@style='cursor:pointer;color:#008000;']/text()")
            for link in external_links:
                item['links'].append(link.extract())
        else:
            name = self.get_text(details, "名稱")
            if not name:
                name = self.get_text(details, "名称")

            vformat = self.get_text(details, "格式")
            if not vformat:
                vformat = self.get_text(details, "格式")

            size = self.get_text(details, "大小")
            if not size:
                size = self.get_text(details, "大小")

            datetime = self.get_text(details, "时长")
            if not datetime:
                datetime = self.get_text(details, "时间")

            imgs = []
            detail_imgs = response.xpath("//div[@class='tpc_content do_not_catch']//img/@data-src")
            for img in detail_imgs:
                imgs.append(img.extract())

            detail_imgs = response.xpath("//div[@class='tpc_content do_not_catch']//img/@ess-data")

            for img in detail_imgs:
                imgs.append(img.extract())

            external_links = response.xpath("//a[@style='cursor:pointer;color:#008000;']/text()")
            for link in external_links:
                item['links'].append(link.extract())

            item['details'].append({
                "name": name,
                "vformat": vformat,
                "time": datetime,
                "size": size,
                "imgs": imgs,
            })
        print(item)

    @staticmethod
    def get_text(li, value):
        for item in li:
            if value in item.extract():
                return item.extract()
        return ""
